{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第五周 推荐系统\n",
    "\n",
    "## 问题描述\n",
    "\n",
    "## 1. 实现基于用户的协同过滤；\n",
    "\n",
    "#### 2. 实现基于物品（活动）的协同过滤；\n",
    "#### 3. 实现基于模型（矩阵分解/LFM）的协同过滤。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 思路：\n",
    "# （1）准备数据：从训练集中，获取用户与活动的评价矩阵，兴趣的为1，不感兴趣的-1，未评价的为-0.1（借鉴同学思路），以及获取活动参与人字典usersForEvent\n",
    "# （2）循环测试集的用户u和需要推荐评价的活动e，\n",
    "#         从活动e参与人usersForEvent[e]得到用户集，用于计算用户相似度。不做是否是朋友的判断，因为这样做了数据太少，其实是需要考虑的，至少是朋友，权重值大\n",
    "#         不做：--从朋友集中，找用户的朋友（认为朋友对活动e都有评价，不再过滤只留下真实有评价的用户），利用“用户--活动”评价矩阵数据，计算当前用户与朋友的相似度\n",
    "# （3）留下相识度>=0.6的朋友，参与推荐评价计算。\n",
    "# （4）推荐评价计算完毕，输出保存评价结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "#保存数据\n",
    "import cPickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of uniqueUsers :3391\n",
      "number of uniqueEvents :13418\n",
      "end for all!\n"
     ]
    }
   ],
   "source": [
    "# （1）准备数据\n",
    "\n",
    "# 统计训练集中有多少不同的用户、活动，为用户、活动创建索引以及评价矩阵准备\n",
    "uniqueUsers = set()\n",
    "uniqueEvents = set()\n",
    "\n",
    "#统计每个用户参加的活动   / 每个活动参加的用户\n",
    "eventsForUser = defaultdict(set)\n",
    "usersForEvent = defaultdict(set)\n",
    "\n",
    "# 文件路径\n",
    "fPath = \"./courseware/\"\n",
    "\n",
    "for filename in [\"train.csv\", \"test.csv\"]:\n",
    "    f = open(fPath + filename, 'rb')\n",
    "    \n",
    "    #忽略第一行（列名字）\n",
    "    f.readline().strip().split(\",\")\n",
    "    \n",
    "    for line in f:    #对每条记录\n",
    "        cols = line.strip().split(\",\")\n",
    "        uniqueUsers.add(cols[0])   #第一列为用户ID\n",
    "        uniqueEvents.add(cols[1])   #第二列为活动ID        \n",
    "       \n",
    "    f.close()\n",
    "\n",
    "\n",
    "n_uniqueUsers = len(uniqueUsers)\n",
    "n_uniqueEvents = len(uniqueEvents)\n",
    "\n",
    "print(\"number of uniqueUsers :%d\" % n_uniqueUsers)\n",
    "print(\"number of uniqueEvents :%d\" % n_uniqueEvents)\n",
    "\n",
    "#用户关系矩阵表，可用于后续LFM/SVD++处理的输入\n",
    "#这是一个稀疏矩阵，记录用户对活动感兴趣\n",
    "userEventScores = ss.dok_matrix((n_uniqueUsers, n_uniqueEvents))\n",
    "\n",
    "# 初始化矩阵，处理未评价的活动，这么初始化太耗内存了，内存飙升到13G，果断暂停，只能用到哪些向量再做局部处理\n",
    "#for i in range(n_uniqueUsers):\n",
    "#    for j in range(n_uniqueEvents):\n",
    "#        userEventScores[i,j] = -0.1\n",
    "\n",
    "#print(\"end of initMatrix\")\n",
    "\n",
    "userIndex = dict()\n",
    "eventIndex = dict()\n",
    "\n",
    "#重新编码用户索引字典\n",
    "for i, u in enumerate(uniqueUsers):\n",
    "    userIndex[u] = i\n",
    "    \n",
    "#重新编码活动索引字典    \n",
    "for i, e in enumerate(uniqueEvents):\n",
    "    eventIndex[e] = i\n",
    "\n",
    "ftrain = open(fPath + \"train.csv\", 'rb')\n",
    "ftrain.readline()\n",
    "for line in ftrain:\n",
    "    cols = line.strip().split(\",\")\n",
    "    i = userIndex[cols[0]]  #用户\n",
    "    j = eventIndex[cols[1]] #活动\n",
    "    \n",
    "    eventsForUser[i].add(j)    #该用户参加了这个活动\n",
    "    usersForEvent[j].add(i)    #该活动被用户参加\n",
    "        \n",
    "    userEventScores[i, j] = int(cols[4]) - int(cols[5])   #interested - not_interested\n",
    "    \n",
    "ftrain.close()\n",
    "\n",
    "  \n",
    "##统计每个用户参加的活动，后续用于将用户朋友参加的活动影响到用户\n",
    "cPickle.dump(eventsForUser, open(\"PE_eventsForUser.pkl\", 'wb'))\n",
    "##统计活动参加的用户\n",
    "cPickle.dump(usersForEvent, open(\"PE_usersForEvent.pkl\", 'wb'))\n",
    "\n",
    "#保存用户-活动关系矩阵R，以备后用\n",
    "sio.mmwrite(\"PE_userEventScores\", userEventScores)\n",
    "\n",
    "#保存用户索引表\n",
    "cPickle.dump(userIndex, open(\"PE_userIndex.pkl\", 'wb'))\n",
    "#保存活动索引表\n",
    "cPickle.dump(eventIndex, open(\"PE_eventIndex.pkl\", 'wb'))\n",
    "\n",
    "print(\"end for all!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:93: RuntimeWarning: invalid value encountered in divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "end for baseUser!\n"
     ]
    }
   ],
   "source": [
    "#相似度/距离\n",
    "import scipy.spatial.distance as ssd\n",
    "\n",
    "# 写文件操作\n",
    "import os # 判断文件是否存在\n",
    "import csv # csv写文件\n",
    "\n",
    "# （2）计算用户相似性\n",
    "userIndex = cPickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "eventIndex = cPickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "\n",
    "# 参加同一个活动的用户集合\n",
    "usersForEvent = cPickle.load(open(\"PE_usersForEvent.pkl\", 'rb'))\n",
    "\n",
    "# 用户参加评价的活动\n",
    "eventsForUser = cPickle.load(open(\"PE_eventsForUser.pkl\", 'rb'))\n",
    "\n",
    "# 用户活动评价矩阵\n",
    "userEventScores = sio.mmread(\"PE_userEventScores\").todense()\n",
    "\n",
    "fwname = \"test_eval_base_user.csv\"\n",
    "if os.path.exists(fwname) == True:\n",
    "    os.remove(fwname)\n",
    "\n",
    "# 准备要写入的文件\n",
    "fw = open(fwname,'wb')\n",
    "writer = csv.writer(fw) # 获取输出数据流\n",
    "\n",
    "# 读取测试集\n",
    "ftest = open(fPath + \"test.csv\", 'rb')\n",
    "#ftest.readline()\n",
    "\n",
    "cols = ftest.readline().strip().split(\",\")\n",
    "cols.append(\"interest\")\n",
    "# print(cols)\n",
    "writer.writerow(cols)\n",
    "\n",
    "for line in ftest:\n",
    "    cols = line.strip().split(\",\")\n",
    "    i = userIndex[cols[0]]  #用户\n",
    "    j = eventIndex[cols[1]] #活动\n",
    "    \n",
    "    \n",
    "    # 与活动j有关的用户\n",
    "    rUser = usersForEvent[j]\n",
    "    if i in rUser: # 从train中获取的数据，应该没有\n",
    "        rUser.remove(i)\n",
    "    n_rUser = len(rUser)\n",
    "    \n",
    "    # 与用户i有关的活动\n",
    "    rEvent = eventsForUser[i]    \n",
    "    # 去掉待评价的活动\n",
    "    if j in rEvent: # 从train中获取的数据，应该没有\n",
    "        rEvent.remove(j)\n",
    "    n_rEvent = len(rEvent)\n",
    "    \n",
    "    # 当前用户对活动的评价矩阵\n",
    "    CR = ss.dok_matrix((1, n_rEvent))\n",
    "    for j2, jj in enumerate(rEvent):        \n",
    "        if userEventScores[i,jj] == 0:\n",
    "            CR[0,j2] = -0.1 # 针对未评价的\n",
    "        else:\n",
    "            CR[0,j2] = userEventScores[i,jj]\n",
    "    \n",
    "    # 临时变量，存放于i,j相关的用户-活动矩阵，用于计算用户相似性\n",
    "    R = ss.dok_matrix((n_rUser, n_rEvent))\n",
    "    \n",
    "    # 临时变量，存放与i用户相关用户的对j的评价值及与i用户的相关系数\n",
    "    # R2 = ss.dok_matrix((n_rUser, 2))\n",
    "    \n",
    "    sumOfsim = 0 # 相似性之和\n",
    "    sumOfProOfevalAndsim = 0 #各用户对j活动评价*相似性之和\n",
    "    \n",
    "    # 将userEventScores相关的评价值，存入R中\n",
    "    for i2,ii in enumerate(rUser):\n",
    "        #R2[i2,0] = userEventScores[ii,j]\n",
    "        for j2, jj in enumerate(rEvent):\n",
    "            if userEventScores[ii,jj] == 0:\n",
    "                R[i2,j2] = -0.1 # 针对未评价的\n",
    "            else:\n",
    "                R[i2,j2] = userEventScores[ii,jj]\n",
    "        \n",
    "        # 计算用户相似性\n",
    "        usim = ssd.correlation(CR.getrow(0).todense(),R.getrow(i2).todense())\n",
    "        if usim >= 0.6:\n",
    "            #R2[i2,1] = usim\n",
    "            # 各用户对j活动评价*相似性之和\n",
    "            sumOfProOfevalAndsim += userEventScores[ii,j] * usim\n",
    "            # 相似性之和\n",
    "            sumOfsim += usim\n",
    "    \n",
    "    # 计算i用户对j活动的评价\n",
    "    aveEval = CR.sum(axis=1)/n_rEvent\n",
    "    \n",
    "    # i用户对j活动的预测评价\n",
    "    evalOfEventByUser = 0\n",
    "    if sumOfsim > 0:\n",
    "        evalOfEventByUser = aveEval + sumOfProOfevalAndsim/sumOfsim\n",
    "       \n",
    "    # 写入文件test_eval中\n",
    "    cols.append(str(evalOfEventByUser))\n",
    "    '''\n",
    "    if evalOfEventByUser > 0.6:\n",
    "        cols.append(\"1\")\n",
    "    else:\n",
    "        cols.append(\"0\")\n",
    "    '''\n",
    "    writer.writerow(cols)\n",
    "\n",
    "fw.close()\n",
    "ftest.close()\n",
    "        \n",
    "print(\"end for baseUser!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 实现基于物品（活动）的协同过滤；"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 思路：\n",
    "# （1）准备数据：从训练集中，获取用户与活动的评价矩阵，兴趣的为1，不感兴趣的-1，未评价的为-0.1（借鉴同学思路），以及获取活动参与人字典usersForEvent\n",
    "# （2）循环测试集的用户u和需要推荐评价的活动e，\n",
    "#         基于上面“基于用户的协同过滤”分析，只转置一下矩阵R，计算余弦相似度即可\n",
    "# （3）留下相识度>=0.6的朋友，参与推荐评价计算。\n",
    "# （4）推荐评价计算完毕，输出保存评价结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "end for baseEvent!\n"
     ]
    }
   ],
   "source": [
    "#相似度/距离\n",
    "import scipy.spatial.distance as ssd\n",
    "\n",
    "# 写文件操作\n",
    "import os # 判断文件是否存在\n",
    "import csv # csv写文件\n",
    "\n",
    "# （2）计算用户相似性\n",
    "userIndex = cPickle.load(open(\"PE_userIndex.pkl\", 'rb'))\n",
    "eventIndex = cPickle.load(open(\"PE_eventIndex.pkl\", 'rb'))\n",
    "\n",
    "# 参加同一个活动的用户集合\n",
    "usersForEvent = cPickle.load(open(\"PE_usersForEvent.pkl\", 'rb'))\n",
    "\n",
    "# 用户参加评价的活动\n",
    "eventsForUser = cPickle.load(open(\"PE_eventsForUser.pkl\", 'rb'))\n",
    "\n",
    "# 用户活动评价矩阵\n",
    "userEventScores = sio.mmread(\"PE_userEventScores\").todense()\n",
    "\n",
    "fwname = \"test_eval_base_event.csv\"\n",
    "if os.path.exists(fwname) == True:\n",
    "    os.remove(fwname)\n",
    "\n",
    "# 准备要写入的文件\n",
    "fw = open(fwname,'wb')\n",
    "writer = csv.writer(fw) # 获取输出数据流\n",
    "\n",
    "# 读取测试集\n",
    "ftest = open(fPath + \"test.csv\", 'rb')\n",
    "#ftest.readline()\n",
    "\n",
    "cols = ftest.readline().strip().split(\",\")\n",
    "cols.append(\"interest\")\n",
    "# print(cols)\n",
    "writer.writerow(cols)\n",
    "\n",
    "for line in ftest:\n",
    "    cols = line.strip().split(\",\")\n",
    "    i = userIndex[cols[0]]  #用户\n",
    "    j = eventIndex[cols[1]] #活动\n",
    "    \n",
    "    \n",
    "    # 与活动j有关的用户\n",
    "    rUser = usersForEvent[j]\n",
    "    if i in rUser: # 从train中获取的数据，应该没有\n",
    "        rUser.remove(i)\n",
    "    n_rUser = len(rUser)\n",
    "    \n",
    "    # 与用户i有关的活动\n",
    "    rEvent = eventsForUser[i]    \n",
    "    # 去掉待评价的活动\n",
    "    if j in rEvent: # 从train中获取的数据，应该没有\n",
    "        rEvent.remove(j)\n",
    "    n_rEvent = len(rEvent)\n",
    "    \n",
    "    # 当前活动各用户的评价矩阵\n",
    "    CR = ss.dok_matrix((1, n_rUser))\n",
    "    for i2, ii in enumerate(rUser):\n",
    "        if userEventScores[ii,j] == 0:\n",
    "            CR[0,i2] = -0.1 # 针对未评价的\n",
    "        else:\n",
    "            CR[0,i2] = userEventScores[ii,j]\n",
    "    \n",
    "    # 临时变量，存放于i,j相关的用户-活动矩阵，用于计算用户相似性\n",
    "    R = ss.dok_matrix((n_rEvent,n_rUser))    \n",
    "       \n",
    "    sumOfsim = 0 # 相似性之和\n",
    "    sumOfProOfevalAndsim = 0 #各用户对j活动评价*相似性之和\n",
    "    \n",
    "    # 将userEventScores相关的评价值，存入R中\n",
    "    for j2,jj in enumerate(rEvent):        \n",
    "        for i2, ii in enumerate(rUser):\n",
    "            if userEventScores[ii,jj] == 0:\n",
    "                R[j2,i2] = -0.1 # 针对未评价的\n",
    "            else:\n",
    "                R[j2,i2] = userEventScores[ii,jj]\n",
    "        \n",
    "        # 计算活动余弦相似性\n",
    "        esim = ssd.cosine(CR.getrow(0).todense(),R.getrow(j2).todense())\n",
    "        if esim >= 0.6:            \n",
    "            # 各用户对j活动评价*相似性之和\n",
    "            sumOfProOfevalAndsim += userEventScores[ii,j] * esim\n",
    "            # 相似性之和\n",
    "            sumOfsim += esim    \n",
    "    \n",
    "    # i用户对j活动的预测评价\n",
    "    evalOfEventByUser = 0\n",
    "    if sumOfsim > 0:\n",
    "        evalOfEventByUser = sumOfProOfevalAndsim/sumOfsim\n",
    "       \n",
    "    # 写入文件test_eval中\n",
    "    cols.append(str(evalOfEventByUser))\n",
    "    '''\n",
    "    if evalOfEventByUser > 0.6:\n",
    "        cols.append(\"1\")\n",
    "    else:\n",
    "        cols.append(\"0\")\n",
    "    '''\n",
    "    writer.writerow(cols)\n",
    "\n",
    "fw.close()\n",
    "ftest.close()\n",
    "\n",
    "print(\"end for baseEvent!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 实现基于模型（矩阵分解/LFM）的协同过滤。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
